{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1cafb0c5-8160-4bba-bafe-28a9d2bab271",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: gensim in e:\\translations\\python\\python311\\site-packages (4.3.3)\n",
      "Requirement already satisfied: numpy<2.0,>=1.18.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (1.26.4)\n",
      "Requirement already satisfied: scipy<1.14.0,>=1.7.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (1.13.1)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from gensim) (5.2.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Skipping C:\\ProgramData\\anaconda3\\Lib\\site-packages\\python_lsp_server-1.7.2.dist-info due to invalid metadata entry 'name'\n",
      "WARNING: Skipping C:\\ProgramData\\anaconda3\\Lib\\site-packages\\python_lsp_server-1.7.2.dist-info due to invalid metadata entry 'name'\n",
      "WARNING: Skipping C:\\ProgramData\\anaconda3\\Lib\\site-packages\\python_lsp_server-1.7.2.dist-info due to invalid metadata entry 'name'\n",
      "WARNING: Skipping C:\\ProgramData\\anaconda3\\Lib\\site-packages\\python_lsp_server-1.7.2.dist-info due to invalid metadata entry 'name'\n"
     ]
    }
   ],
   "source": [
    "pip install gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5c5a740a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pprint\n",
    "\n",
    "from gensim.models import KeyedVectors\n",
    "\n",
    "# 从GloVe官网下载GloVe向量，此处使用的是glove.6B.zip\n",
    "# 解压缩zip文件并将以下路径改为解压后对应文件的路径\n",
    "model = KeyedVectors.load_word2vec_format('F:\\BaiduNetdiskDownload\\glove.6B'+\\\n",
    "    '/glove.6B.100d.txt', binary=False, no_header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "01a2e4a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('movie', 0.9055122137069702),\n",
      " ('films', 0.8914434909820557),\n",
      " ('directed', 0.8124362230300903),\n",
      " ('documentary', 0.8075793981552124),\n",
      " ('drama', 0.7929168939590454),\n",
      " ('movies', 0.7889865040779114),\n",
      " ('comedy', 0.7842751145362854),\n",
      " ('starring', 0.7573285698890686),\n",
      " ('cinema', 0.7419456839561462),\n",
      " ('hollywood', 0.7307389378547668)]\n",
      "[('vehicle', 0.8630837798118591),\n",
      " ('truck', 0.8597877025604248),\n",
      " ('cars', 0.837166965007782),\n",
      " ('driver', 0.8185910582542419),\n",
      " ('driving', 0.781263530254364),\n",
      " ('motorcycle', 0.7553157210350037),\n",
      " ('vehicles', 0.7462257146835327),\n",
      " ('parked', 0.74594646692276),\n",
      " ('bus', 0.737270712852478),\n",
      " ('taxi', 0.7155268788337708)]\n"
     ]
    }
   ],
   "source": [
    "# 使用most_similar()找到词表中距离给定词最近（最相似）的n个词\n",
    "pprint.pprint(model.most_similar('film'))\n",
    "pprint.pprint(model.most_similar('car'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8b62f7ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "japanese\n",
      "panda\n",
      "longest\n",
      "terrible\n",
      "queen\n"
     ]
    }
   ],
   "source": [
    "# 利用GloVe展示一个类比的例子\n",
    "def analogy(x1, x2, y1):\n",
    "    # 寻找top-N最相似的词。\n",
    "    result = model.most_similar(positive=[y1, x2], negative=[x1])\n",
    "    return result[0][0]\n",
    "\n",
    "print(analogy('china', 'chinese', 'japan'))\n",
    "print(analogy('australia', 'koala', 'china'))\n",
    "print(analogy('tall', 'tallest', 'long'))\n",
    "print(analogy('good', 'fantastic', 'bad'))\n",
    "print(analogy('man', 'woman', 'king'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c308cee",
   "metadata": {},
   "source": [
    "下面将展示word2vec的代码，包括文本预处理、skipgram算法的实现、以及使用PyTorch进行优化。这里使用《小王子》这本书作为训练语料。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f51b6177-2285-4a84-9e61-30882e9a59d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to E:\\Translations\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "590fc408",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 安装NLTK，使用如下代码下载punkt组件\n",
    "#import nltk\n",
    "#nltk.download('punkt')\n",
    "\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from collections import defaultdict\n",
    "\n",
    "# 使用类管理数据对象，包括文本读取、文本预处理等\n",
    "class TheLittlePrinceDataset:\n",
    "    def __init__(self, tokenize=True):\n",
    "        # 利用NLTK函数进行分句和分词\n",
    "        text = open('the little prince.txt', 'r', encoding='utf-8').read()\n",
    "        if tokenize:\n",
    "            self.sentences = sent_tokenize(text.lower())\n",
    "            self.tokens = [word_tokenize(sent) for sent in self.sentences]\n",
    "        else:\n",
    "            self.text = text\n",
    "\n",
    "    def build_vocab(self, min_freq=1):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for sentence in self.tokens:\n",
    "            for token in sentence:\n",
    "                frequency[token] += 1\n",
    "        self.frequency = frequency\n",
    "\n",
    "        # 加入<unk>处理未登录词，加入<pad>用于对齐变长输入进而加速\n",
    "        self.token2id = {'<unk>': 1, '<pad>': 0}\n",
    "        self.id2token = {1: '<unk>', 0: '<pad>'}\n",
    "        for token, freq in sorted(frequency.items(), key=lambda x: -x[1]):\n",
    "            # 丢弃低频词\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "            else:\n",
    "                break\n",
    "\n",
    "    def get_word_distribution(self):\n",
    "        distribution = np.zeros(vocab_size)\n",
    "        for token, freq in self.frequency.items():\n",
    "            if token in dataset.token2id:\n",
    "                distribution[dataset.token2id[token]] = freq\n",
    "            else:\n",
    "                # 不在词表中的词按<unk>计算\n",
    "                distribution[1] += freq\n",
    "        distribution /= distribution.sum()\n",
    "        return distribution\n",
    "\n",
    "    # 将分词结果转化为索引表示\n",
    "    def convert_tokens_to_ids(self, drop_single_word=True):\n",
    "        self.token_ids = []\n",
    "        for sentence in self.tokens:\n",
    "            token_ids = [self.token2id.get(token, 1) for token in sentence]\n",
    "            # 忽略只有一个token的序列，无法计算loss\n",
    "            if len(token_ids) == 1 and drop_single_word:\n",
    "                continue\n",
    "            self.token_ids.append(token_ids)\n",
    "        \n",
    "        return self.token_ids\n",
    "\n",
    "dataset = TheLittlePrinceDataset()\n",
    "dataset.build_vocab(min_freq=1)\n",
    "sentences = dataset.convert_tokens_to_ids()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e22d24ab-de59-41cb-8020-b2177924f409",
   "metadata": {},
   "source": [
    "小王子为部分摘录"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "efc882de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1422, 2) [[ 1  1]\n",
      " [ 1 36]\n",
      " [ 1  1]\n",
      " ...\n",
      " [ 1  4]\n",
      " [ 4 64]\n",
      " [ 4  1]]\n"
     ]
    }
   ],
   "source": [
    "# 遍历所有的中心词-上下文词对\n",
    "window_size = 2\n",
    "data = []\n",
    "\n",
    "for sentence in sentences:\n",
    "    for i in range(len(sentence)):\n",
    "        for j in range(i-window_size, i+window_size+1):\n",
    "            if j == i or j < 0 or j >= len(sentence):\n",
    "                continue\n",
    "            center_word = sentence[i]\n",
    "            context_word = sentence[j]\n",
    "            data.append([center_word, context_word])\n",
    "\n",
    "# 需要提前安装numpy\n",
    "import numpy as np\n",
    "data = np.array(data)\n",
    "print(data.shape, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "30903b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 需要提前安装PyTorch\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# 实现skipgram算法，使用对比学习计算损失\n",
    "class SkipGramNCE(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_size, distribution,\\\n",
    "                 neg_samples=20):\n",
    "        super(SkipGramNCE, self).__init__()\n",
    "        print(f'vocab_size = {vocab_size}, embed_size = {embed_size}, '+\\\n",
    "              f'neg_samples = {neg_samples}')\n",
    "        self.input_embeddings = nn.Embedding(vocab_size, embed_size)\n",
    "        self.output_embeddings = nn.Embedding(vocab_size, embed_size)\n",
    "        distribution = np.power(distribution, 0.75)\n",
    "        distribution /= distribution.sum()\n",
    "        self.distribution = torch.tensor(distribution)\n",
    "        self.neg_samples = neg_samples\n",
    "        \n",
    "    def forward(self, input_ids, labels):\n",
    "        i_embed = self.input_embeddings(input_ids)\n",
    "        o_embed = self.output_embeddings(labels)\n",
    "        batch_size = i_embed.size(0)\n",
    "        n_words = torch.multinomial(self.distribution, batch_size * \\\n",
    "            self.neg_samples, replacement=True).view(batch_size, -1)\n",
    "        n_embed = self.output_embeddings(n_words)\n",
    "        pos_term = F.logsigmoid(torch.sum(i_embed * o_embed, dim=1))\n",
    "        # 负采样，用于对比学习\n",
    "        neg_term = F.logsigmoid(- torch.bmm(n_embed, \\\n",
    "            i_embed.unsqueeze(2)).squeeze())\n",
    "        neg_term = torch.sum(neg_term, dim=1)\n",
    "        loss = - torch.mean(pos_term + neg_term)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1d9da6c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.         0.15328467 0.06812652 0.06812652 0.06569343 0.06082725\n",
      " 0.0486618  0.03163017 0.02676399 0.02676399 0.0243309  0.02189781\n",
      " 0.01703163 0.01703163 0.01703163 0.01703163 0.01459854 0.01459854\n",
      " 0.01459854 0.01459854 0.01216545 0.01216545 0.00973236 0.00973236\n",
      " 0.00973236 0.00973236 0.00729927 0.00729927 0.00729927 0.00729927\n",
      " 0.00729927 0.00729927 0.00729927 0.00729927 0.00729927 0.00729927\n",
      " 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618\n",
      " 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618\n",
      " 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618\n",
      " 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618\n",
      " 0.00486618 0.00486618 0.00486618 0.00486618 0.00486618]\n",
      "vocab_size = 65, embed_size = 128, neg_samples = 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-99, loss=4.1295: 100%|█| 100/100 [00:02<00:00, 44.03it\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGwCAYAAABPSaTdAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAABrV0lEQVR4nO3dd3hb5dk/8O+RZEke8rZlO7ZjZ5C9p5MwCi4JYSQBymgoYaaUUAiU2ZbwlhVWIYwUCqVhlFV+QFhNIAQI2Tshiyw7sTO8hyx5yJLO7w/pHEu2ZEu2NfP9XJevN5aO5BO9gL+9n/u5H0EURRFEREREEUoR7BsgIiIi8ieGHSIiIopoDDtEREQU0Rh2iIiIKKIx7BAREVFEY9ghIiKiiMawQ0RERBFNFewbCAU2mw2nTp2CTqeDIAjBvh0iIiLygiiKaGhoQFZWFhQKz/Ubhh0Ap06dQk5OTrBvg4iIiLqhtLQU2dnZHp9n2AGg0+kA2D+s+Pj4IN8NERERecNgMCAnJ0f+Pe4Jww4gL13Fx8cz7BAREYWZrlpQ2KBMREREEY1hh4iIiCIaww4RERFFNIYdIiIiimgMO0RERBTRGHaIiIgoojHsEBERUURj2CEiIqKIxrBDREREES2oYeenn37CpZdeiqysLAiCgOXLl7s8L4oiFi1ahMzMTERHR6OwsBCHDx92uaampgZz585FfHw8EhMTcfPNN8NoNAbwb0FEREShLKhhx2QyYdSoUVi6dKnb55955hm89NJLeO2117B582bExsZi+vTpaG5ulq+ZO3cu9u3bh1WrVuGrr77CTz/9hPnz5wfqr0BEREQhThBFUQz2TQD2cy0+++wzzJ49G4C9qpOVlYU//elPuPfeewEA9fX10Ov1eOutt3DNNdfgwIEDGDp0KLZu3Yrx48cDAFauXImZM2fixIkTyMrKcvuzWlpa0NLSIn8vHSRWX1/Ps7GIiIjChMFgQEJCQpe/v0O2Z6e4uBhlZWUoLCyUH0tISMCkSZOwceNGAMDGjRuRmJgoBx0AKCwshEKhwObNmz2+9+LFi5GQkCB/5eTk+OXvUNdoRlGlEc2tVr+8PxEREXUtZMNOWVkZAECv17s8rtfr5efKysqQnp7u8rxKpUJycrJ8jTsPPfQQ6uvr5a/S0tJevnu7mS+uxfl/X4ODZQ1+eX8iIiLqmirYNxAMGo0GGo3G7z8nKVaNU/XNqGk0+/1nERERkXshW9nJyMgAAJSXl7s8Xl5eLj+XkZGBiooKl+ctFgtqamrka4IpOVYNAKg1MewQEREFS8iGnfz8fGRkZGD16tXyYwaDAZs3b0ZBQQEAoKCgAHV1ddi+fbt8zffffw+bzYZJkyYF/J7bS4qxh50ahh0iIqKgCeoyltFoxJEjR+Tvi4uLsWvXLiQnJyM3NxcLFy7E448/joEDByI/Px8PP/wwsrKy5B1bQ4YMwYwZM3DrrbfitddeQ2trK+644w5cc801HndiBVJSTBQAoJbLWEREREET1LCzbds2/OpXv5K/v+eeewAA8+bNw1tvvYX7778fJpMJ8+fPR11dHaZNm4aVK1dCq9XKr3nvvfdwxx134IILLoBCocAVV1yBl156KeB/F3eSYqXKTmuQ74SIiOjMFTJzdoLJ2336vnpn4zEs+nwfLhqegVevG9dr70tEREQRMGcnErBnh4iIKPgYdvxI3o3Fnh0iIqKgYdjxo7bKDnt2iIiIgoVhx4+cKztsjSIiIgoOhh0/SnRsPbfaRBiaLUG+GyIiojMTw44faaOUiFErAdgPBSUiIqLAY9jxM+7IIiIiCi6GHT/jjiwiIqLgYtjxM05RJiIiCi6GHT9Lls7H4jIWERFRUDDs+Jlc2eEyFhERUVAw7PiZ1KDMyg4REVFwMOz4WRIblImIiIKKYcfPkuXKDhuUiYiIgoFhx8+SYu0NyuzZISIiCg6GHT+T5+ywZ4eIiCgoGHb8TF7GajTDZuNhoERERIHGsONniY6wYxMBQzP7doiIiAKNYcfP1CoF4jQqAEBtI8MOERFRoDHsBIDcpMy+HSIiooBj2AmAZA4WJCIiChqGnQDgkRFERETBw7ATAKzsEBERBQ/DTgCwskNERBQ8DDsBkBRjb1Cu45ERREREAcewEwCs7BAREQWPKtg3cCbwpWfHahPxyBd7odNG4eZp+UiN0/j79oiIiCIaw04A+FLZOXDagP9sKgEAvLX+GOZNycP8c/rJZ2wRERGRb7iMFQC+HAZaaWyR/9zUasVra47i7Ke/x3PfHERzq9Vv90hERBSpGHYCIMmxjFXX1AprF4eB1hjtgejsgal4c954DMuKh8lsxSs/HMFHW0v9fq9ERESRhmEnABIdu7FEEahv6nxHVq1jqSs5Vo0Lhujx1R+n4arx2QCA0ppG/94oERFRBGLYCYAopQI6rXQYaOdLWdWmtrADAIIg4Cy9DgBQ3tDi8XVERETkHsNOgHjbtyMtY0k7uABAH68FAJQbmv10d0RERJGLYSdApL6drk4+lys7cR3DTgXDDhERkc8YdgJErux0sYxVY7IvVaU4bTVP19ln7ZQbWiCKnTc4ExERkSuGnQBpq+x01aBsfz45tm2YYHq8/c9NrVYYWyx+ukMiIqLIxLATIMmx9h1ZXTYoO+bsOA8RjFGr5AbncgOblImIiHzBsBMgiV4cGdFqtcHQbK/ctJ+YzL4dIiKi7mHYCRBvenakIKQQgMToKJfn9I6lrPIGhh0iIiJfMOwEiDe7saSdWEkxaigUgstz6Tpp+zmXsYiIiHzBsBMgbZUdzw3KNe0GCjqTmpQrGHaIiIh8wrATIFKDcmeVnc7Cjl6q7HAZi4iIyCcMOwEiLWPVN7XCYrW5vUYKOylxbsIOG5SJiIi6hWEnQBKioyA42nA8HQbq3LPTntygzGUsIiIinzDsBIhKqUC8tvNZO+6mJ0ucz8fiFGUiIiLvMewEkNSL42mKcmc9O2mOIyNaLG2zeIiIiKhrDDsBlBTTeZNytXTieZymw3PaKCUSHLN32LdDRETkPYadAOpqsKD0uLtlLIB9O0RERN3BsBNAXQ0WrOmkQRlw7dshIiIi7zDsBJBc2XETdmw2UR446G7rOeA0RZmzdoiIiLzGsBNASZ1MUa5vaoXVZt9l5amywynKREREvmPYCSCpQdldz440Y0enVUGtcv//Fr1jR1YFKztEREReY9gJIKliU23sWJmRpyd7aE4GnHt2WNkhIiLyFsNOAOUkxwAAjtc0dniusxk7knQ2KBMREfmMYSeA8lJiAQB1ja0dmpS9CTt6p54dTlEmIiLyDsNOAEWrlchKsFdniqpMLs9JR0V0FnakKcpmq83j+VpERETkimEnwPLT7NWd4nZhp1qu7HScnizRqJRykzP7doiIiLzDsBNg+alS2DG6PO5NgzLAwYJERES+CumwY7Va8fDDDyM/Px/R0dHo378/HnvsMZd+FVEUsWjRImRmZiI6OhqFhYU4fPhwEO+6c/mpcQA6Vna86dkB2KRMRETkq5AOO08//TReffVVvPLKKzhw4ACefvppPPPMM3j55Zfla5555hm89NJLeO2117B582bExsZi+vTpaG4OzTDQz1HZKarsXthpm7XDZSwiIiJvqIJ9A53ZsGEDZs2ahYsvvhgAkJeXhw8++ABbtmwBYK/qLFmyBH/9618xa9YsAMA777wDvV6P5cuX45prrgnavXsiLWMdqzbBZhOhUAgAfAg7jsoOTz4nIiLyTkhXdqZMmYLVq1fj0KFDAIDdu3dj3bp1uOiiiwAAxcXFKCsrQ2FhofyahIQETJo0CRs3bvT4vi0tLTAYDC5fgZKdFI0opYDmVhtOOwKLKIpODcpdLWPx5HMiIiJfhHRl58EHH4TBYMDgwYOhVCphtVrxxBNPYO7cuQCAsrIyAIBer3d5nV6vl59zZ/Hixfjb3/7mvxvvhEqpQG5yDI5WmlBcaUKfxGiYzFaYLTYAng8BlfAwUCIiIt+EdGXnv//9L9577z28//772LFjB95++20899xzePvtt3v0vg899BDq6+vlr9LS0l66Y++0NSnbd2TVGO1VHW2UAjHqzvOnnoeBEhER+SSkKzv33XcfHnzwQbn3ZsSIETh+/DgWL16MefPmISMjAwBQXl6OzMxM+XXl5eUYPXq0x/fVaDTQaDzPs/G3fmmxwIG2wYLV0kBBD6edO5N7dhqaXXp+iIiIyL2Qruw0NjZCoXC9RaVSCZvNvuSTn5+PjIwMrF69Wn7eYDBg8+bNKCgoCOi9+qJt1o497EinoCd3sYQFtE1RbrWKbk9PJyIiIlchXdm59NJL8cQTTyA3NxfDhg3Dzp078fzzz+Omm24CAAiCgIULF+Lxxx/HwIEDkZ+fj4cffhhZWVmYPXt2cG++E+3DTrWx6+nJkiilAimxalSbzKhoaEFKXPAqVEREROEgpMPOyy+/jIcffhi33347KioqkJWVhd///vdYtGiRfM39998Pk8mE+fPno66uDtOmTcPKlSuh1WqDeOedk2btlNY0wmyxeT09WZIer0W1yYxyQzOGZMb77T6JiIgiQUiHHZ1OhyVLlmDJkiUerxEEAY8++igeffTRwN1YD6XpNIhVK2EyW1FS0+j1jB2JPl6DA6fZpExEROSNkO7ZiVSCIKBfmn1HVlGl0esZOxK9jkdGEBEReYthJ0ic+3a6U9kBgjtrZ9uxGpTWNAbt5xMREXmLYSdIehJ20uUjI4KzjFVS3Yjf/HMjbn1nW1B+PhERkS8YdoKkX5rjQFCnsON1g7JOquwEJ+zsPVUPUQRO1DYF5ecTERH5IqQblCOZc2WnyWwF4MsyVnAPA5W2zBtbLLDaRCg52JCIiEIYKztBkucIO5UNLTC2WAAAKV7M2QGcpyi3wGYT/XODnThaaZT/LN07ERFRqGLYCZJ4bRRSnQYCKhUCdFrvCm1JsVEAAKtNREMQwoZU2QEYdoiIKPQx7ASRNFwQAJJi1F6fc6VRKaGNsv+/ztDU6pd764xz2GloDvzPJyIi8gXDThDlO4Udb5uTJQnR9upOfYDDTo3JjLrGtp/Z0MzKDhERhTaGnSCSdmQB3jcnSxKj7dcHOuwUVxldvmdlh4iIQh3DThA5V3a8OfHcmVTZca6yBEJRpcnle1Z2iIgo1DHsBJFLZSfGt7ATH6RlrKIqhh0iIgovDDtBlJMcA6kn2ddlrGD17BSzskNERGGGYSeINColspNiAAAp3VzGCnzPjj3sSDvJ2LNDREShjmEnyEbnJAIABjhOQfdWMMKO1SaiuNoedkZmJwDgnB0iIgp9PC4iyJ6YMxw3TcvHKEd48FZijD3sBHLOzqm6JpgtNqhVCgzKiAdwistYREQU8hh2gkynjZKrO76Qd2M1mXv5jjyTmpPzUmLkn89lLCIiCnVcxgpTwVjGKnaciZWfGisfbWFgZYeIiEIcw06YCsbWc6k5OT81Tg47RoYdIiIKcQw7YUqu7ARwqKC0jNUvra2y09DCZSwiIgptDDthSu6ZabHAZhMD8jOl6cn9UmOh00o9O6zsEBFRaGPYCVNS2BHFwASO5lYrTtU3AXDt2TE2WyCKgQlbRERE3cGwE6bUKgVi1EoAgdmRdazaBFG0h6zkWLVc2bHYRDS32vz+84mIiLqLYSeMBXJHlnRMRH5qLARBQEyUEoLjqAtuPyciolDGsBPGAhl2itodE6FQCIjTcPs5ERGFPoadMBbI7edyc7LTSe3xjqUsHhlBREShjGEnjAV0GatKGijYdoaXvP2cy1hERBTCGHbCWGIwlrGcKjvSMha3nxMRUShj2AljgRosWGsyo87xM/JS2sIOKztERBQOGHbCWKCWsaSqTlaCFtGO7e4AOFiQiIjCAsNOGEuICVDYcRwA2i8tzuXxtsoOww4REYUuhp0wFqjKTtsBoLEuj8cx7BARURhg2Aljgdp67insxMvLWOzZISKi0MWwE8YCtRur3NAMAMhK1Lo8Lp+PxTk7REQUwhh2wligdmOZWqwA2hqSJezZISKicMCwE8aksNPQYoHV5r+Tx6XKjTRXRxKn4TIWERGFPoadMCb17ACAwY9LWVKYkRqSJazsEBFROGDYCWNRSgViHXNv/NW3I4qiXNnRaTyEHfbsEBFRCGPYCXOJMWoA/gs7Ta1WSCtk7Ss73I1FREThgGEnzPl7+7nRsUSlEIDoKKXLc1IPT3OrDa1Wm19+PhERUU8x7IS5hGh74KjzU9iRlqhiNSoIguDynHOlh307REQUqhh2wpy/pyhLlZ32/TqAvWdIqvYYGXaIiChEMeyEOSns+Gs3lrztXNsx7ABtTcoG9u0QEVGIYtgJc36v7HiYsSPh+VhERBTqGHbCnLwby09TlKXlqbh205MlOu7IIiKiEMewE+b8vhvLw4wd+efzfCwiIgpxDDthTlrGqmsy++X9u1rG4hRlIiIKdQw7Ya6tZ8c/YaOhufMGZSkEcRmLiIhCFcNOmPP/biz7+8Z6rOxIPTus7BARUWhi2AlzwZyzA/B8LCIiCn0MO2Eu0RF2jC0WWPxwZEPXc3ZY2SEiotDGsBPmpN1YAGDwQ+DoskGZPTtERBTiGHbCnFIhyIGjrrH3d2R5O0GZx0UQEVGoYtiJAP6ctdN1zw6XsYiIKLQx7EQAfzYpe1vZ4TIWERGFKoadCODPsCPP2eHZWEREFKYYdiJAYox/Zu2YLTa0WOw7vLqaoGw0W2Czib3684mIiHoDw04E8Fdlx+Q0O8fTUMF4R8+OKAImM6s7REQUehh2IoB8PlYvn3wu9etooxSIUrr/R0WjUiBKKQDgUhYREYWmkA87J0+exHXXXYeUlBRER0djxIgR2LZtm/y8KIpYtGgRMjMzER0djcLCQhw+fDiIdxx4/tqN1TZjJ8rjNYIgOJ2PxbBDREShJ6TDTm1tLaZOnYqoqCisWLEC+/fvx9///nckJSXJ1zzzzDN46aWX8Nprr2Hz5s2IjY3F9OnT0dzcHMQ7D6yeLGMVV5mw+kC52+eksKPzsBNLIm0/l87RIiIiCiWd/xYLsqeffho5OTlYtmyZ/Fh+fr78Z1EUsWTJEvz1r3/FrFmzAADvvPMO9Ho9li9fjmuuuSbg9xwMPQk7f/xgB/aeNOCbhedgUIbO5TljFzuxJFIY8scEZyIiop4K6crOF198gfHjx+M3v/kN0tPTMWbMGLzxxhvy88XFxSgrK0NhYaH8WEJCAiZNmoSNGzd6fN+WlhYYDAaXr3Am7cbyNeyIooiiShMA4Hi1qcPzDV0cFSHRcfs5ERGFsJAOO0VFRXj11VcxcOBAfPPNN/jDH/6AO++8E2+//TYAoKysDACg1+tdXqfX6+Xn3Fm8eDESEhLkr5ycHP/9JQJAquz4uvXc2GJBo9kKAKgxdTxqQq7sdLGMJfX0cLAgERGFopAOOzabDWPHjsWTTz6JMWPGYP78+bj11lvx2muv9eh9H3roIdTX18tfpaWlvXTHwSHvxvIx7JQbWuQ/V7sLO44enK4qO/E8H4uIiEJYSIedzMxMDB061OWxIUOGoKSkBACQkZEBACgvd22wLS8vl59zR6PRID4+3uUrnElhp9FsRavV5vXrKgxtTdydVna4jEVERGEspMPO1KlTcfDgQZfHDh06hL59+wKwNytnZGRg9erV8vMGgwGbN29GQUFBQO81mKTdUIBvfTvlDZ2HnYYuzsVq//O5jEVERKEopMPO3XffjU2bNuHJJ5/EkSNH8P777+P111/HggULANhnvCxcuBCPP/44vvjiC+zZswfXX389srKyMHv27ODefAApFYJcXfEp7HSxjGXyskGZ52MREVEoC+mt5xMmTMBnn32Ghx56CI8++ijy8/OxZMkSzJ07V77m/vvvh8lkwvz581FXV4dp06Zh5cqV0Gq1QbzzwEuMiUJDs8XHsONc2Wnp8Lz3c3YcYaeFYYeIiEJPSIcdALjkkktwySWXeHxeEAQ8+uijePTRRwN4V6EnIToKpWjyKexUOFV2aoxulrG87tnhMhYREYWukA875B15sKDjfKzKhhb8a10RRBF46KLBEAShw2ucKzvVJjNEUXS5zsg5O0REFAEYdiKEFHZKaxrx3DcH8e/1xfIMnasn5KB/WlyH1zg3KLdYbGg0W11ON/d2zo6OZ2MREVEIY9iJEFLY+fuqQx2eK6lp7BB2RFF0aVAG7DuyXMKO15Ud6Wws38PO1mM1MDS14oIh+q4vJiIi6oaQ3o1F3kuKUct/Pksfh3/+bhwKHQHiRE1jh+vrm1phtthn8qTG2V/bfkeW73N2WiGKotf3LIoibnl7G259ZxvK6s+cg1uJiCiwWNmJEFeOy8bxmkZcMDgds0b3gVIhYEtxDQCgtLapw/VSVScpJgoZCVpUGc0uO7JEUYTR7O2cHfvzrVYRLRYbtFFKr+65vqlVbqj+pcyAjIQzawcdEREFBsNOhOiXFoelvx3r8lhOUjQAoKS6Y2WnzNGcrI/XIjlWAwCodtqR1Wi2QirS6DRRHV7vLFatgiAAoggYmlu9DjvOy2hHKow4b1C6V68jIiLyBZexIlhOcgwAoLS2Y9iRdmKlx2uREmtfxnKeoiz13ygVArRRnf9jolAIiFP7fj5WhVOD9OFyo9evIyIi8gXDTgSTw46bnh3pXCy9ToNkN2HHecaOu23r7XVn+7nznJ/DFQ1ev46IiMgXDDsRLCfJHnYMzRZ5/o5EWkKyL2N1bFD2dieWpG2woC+VHeewY/SpuZmIiMhbDDsRLFqtRGqcvR+n/VJWudyzo3G/jNXs3VERkjinHVnecl7Gami2uIQfIiKi3sKwE+Fyku1Nyu2XssodwSLdY2XHHlpiva7s+H4+Vvtww74dIiLyB4adCCctZbWv7FQ47cZKiZMqO23hw9tzsSTSMpbBh7O5Kh1LaVFKe08Q+3aIiMgfGHYiXK6jSbnEqbJjs4lyVSXDaeu582Ggcs+Ol8tY7pbCuiIdVzEmNwmAvW+HiIiotzHsRLi2Zay2wYLVJjOsNhGCYJ+eLC1jmcxWNLfaz9MyOcKOzsvKjjSFucroXd+NKIrybqyp/VMBAEe4jEVERH7QrbDz9ttv4+uvv5a/v//++5GYmIgpU6bg+PHjvXZz1HPulrGk5uTUOA1USgXitSp5KUmqzDT4uBsrTWevDlV62WRsbLGgyRGspg5IAQAcqmjgjiwiIup13Qo7Tz75JKKj7RWDjRs3YunSpXjmmWeQmpqKu+++u1dvkHpGmrVzoqYJNps9SEi7oPTx9oAiCIJ8tpYUdrw98Vwi7fqqMnq3jCUto8VpVBjeJwGCANQ1tnY4n4uIiKinuhV2SktLMWDAAADA8uXLccUVV2D+/PlYvHgx1q5d26s3SD2TmaCFUiHAbLXJAUOesaNrO4uq/Y4sX+fs+FrZkZaw0nUaaKOUcm+Rux1ZRyuN+P6Xcq/el4iIqL1uhZ24uDhUV1cDAL799lv8+te/BgBotVo0NXU8dJKCR6VUICvRHmqkpSznoyIk7Xdk+TpnR6rsVJta5ApSZ6TqkhSSBqbHAQCOtNuRZbOJuP7NLbjprW04XM7dWkRE5LtuhZ1f//rXuOWWW3DLLbfg0KFDmDlzJgBg3759yMvL6837o14g9+3USGFHmp6ska9JaXcYqNSz4+2cHSkstVpF+STzzlQ4TXAGgAHpOgAdd2TtKKnFyTp7gD7u5kBTIiKirnQr7CxduhQFBQWorKzEJ598gpQUe4Pp9u3bce211/bqDVLPtd9+7jxjR9JhGcvHOTsalRIJ0fZZO97syJIqO+ntKjvtl7G+3nNa/rMv29qJiIgk3v0maycxMRGvvPJKh8f/9re/9fiGqPe1HQhqr5CUGVwblAGnOTlG154db5exAPuSVH1TKyobWjBQr+v02gp5grMj7OgdYcepsmOziVixp0z+vqaRYYeIiHzXrcrOypUrsW7dOvn7pUuXYvTo0fjtb3+L2traXrs56h3ZSY5ZO7Wuy1jpzg3Kca6VHZPcoBzl9c+RZu1UelPZaXcP/dPsYafK2IJaxz3sLK2VgxnAyg4REXVPt8LOfffdB4PBAADYs2cP/vSnP2HmzJkoLi7GPffc06s3SD3Xtv28Ea1WG6pNrv0ygPMEZPtzDT5OUAaANEdw8WZHVvtlrFiNCn0S7aHsSKW9uvP1z/aqjmAfAST3ExEREfmiW2GnuLgYQ4cOBQB88sknuOSSS/Dkk09i6dKlWLFiRa/eIPWc1LNz2tCMU3VNEEVAqRDkgAOg7cgIkxktFivMFhsA73t2AOcpyl2HkvbLWIDTUla50b6Etdfer3PB4HQAQC2XsYiIqBu6FXbUajUaG+1LIt999x0uvPBCAEBycrJc8aHQkRKrRnSUEqII7CypA2CvqCgUgnyNc4OyqcUqP+5L2PF21k6T2SofNOq8/V1uUq5owM7SOpyub0asWolZo/vI90ZEROSrbjUoT5s2Dffccw+mTp2KLVu24KOPPgIAHDp0CNnZ2b16g9RzgiAgJzkah8qN2Ha8BoBryADalrEami1yb0yMWgmlUyDqStsU5c7DjrSEpY1SuJy9NdCx/fxIhRH/c+zCumCIXp4T5HwqOxERkbe6Vdl55ZVXoFKp8P/+3//Dq6++ij597P/Le8WKFZgxY0av3iD1DmnWzrZj9gZyvU7j8nxCdJQcbKR5PN7O2JF4W9mRl7B0WghCW5ga4FjGOljWgBWOsDNzRKa8xFZr6np+DxERUXvdquzk5ubiq6++6vD4Cy+80OMbIv+QmpQPOqYQ69tVdhQKAUkxUagymnG82gTA+xPPJWneVnacjopwNsCxjCWFoVi1EucNSkNLq71/yNhiQYvFCo1K6dN9ERHRma1bYQcArFYrli9fjgMHDgAAhg0bhssuuwxKJX8RhSIp7EiHimckaDtckxyrtocdR2XHl51YQFtlp9pkhs0muvQEOZN3YsW7hp14bRQy4rXydvPzh+ihjVJCo1JApRBgsYmoMZmRmRDt030REdGZrVth58iRI5g5cyZOnjyJQYMGAQAWL16MnJwcfP311+jfv3+v3iT1XE6Sa0BoX1UB2pqUpWUsX5qTnV9vtYmobTQjJa7jzwBcl7HaG6iPk8POxSMyADhOZY9Vo7KhBdVGhh0iIvJNt3p27rzzTvTv3x+lpaXYsWMHduzYgZKSEuTn5+POO+/s7XukXpCbEuPyfftlLKDtfCzpDCpfw06UUiEHns4GC8rLWPEdw5C0lBWjVuK8Qeny48kx9vfl9nMiIvJVt8LOmjVr8MwzzyA5OVl+LCUlBU899RTWrFnTazdHvUdqUJa4CztSUCnp5jIW4DRrp8FzKGkbKNjxHibm2f+ZunhEJrRRbUuiyfLQQ4YdIiLyTbeWsTQaDRoaGjo8bjQaoVar3byCgi1Wo0JyrFoOC3o3VRUpULQ4Bgr62qAM2Pt2DpUbUWls9niNpwZlAJgxPAOf/GEKhmXFu96bdJwFpygTEZGPulXZueSSSzB//nxs3rwZoihCFEVs2rQJt912Gy677LLevkfqJVLfjlqlkE8od5YS5xpUu1PZkXdkeVPZcRO4BEHAuL5JLlUdoG0OEJexiIjIV90KOy+99BL69++PgoICaLVaaLVaTJkyBQMGDMCSJUt6+Rapt0g7svTxGpf5NpLkWNew4+ucHaBtsKCnnh2zxYbaRvu8HHfLWJ4kxbgeVEpEROStbi1jJSYm4vPPP8eRI0fkredDhgzBgAEDevXmqHfJYcdDyGgfdrq7jAUAVR4GC0ohKEppn+vjLanqVMNlLCIi8pHXv826Os38hx9+kP/8/PPPd/+OyG+GZtr7YKQdT+1Ju7Ek3WtQ7ryyU2Foa052V13yRG5Q5jIWERH5yOvfZjt37vTqOl9+gVFgzRyRifjoKIzOSXT7fPvKTpzG+8qLpKsjI6QZO2lumpM7I209524sIiLylddhx7lyQ+FJqRBw7llpHp9vv6zk65wdoOvDQNsqOz6GnTiGHSIi6p5uNShTZFIpFUh0Cjy67uzGcjoywmK1dXhenp7sZidWZ5KddmNZbaLP90VERGcuhh1y4byU1Z3KTnKsGgrBfgaXu/6athk73u/EAtp2Y4kiUN/E08+JiMh7DDvkIsU57HSjsqNUCEiO9dy30zY92bfKTpRSgXjH/dSYOj9VnYiIyBnDDrnoaWUHcDoyws028e4uYwGQDxblFGUiIvIFww65kKoyKoUAjap7/3h0tiOrsxPPu743TlEmIiLfMeyQC2kZK06r6vYYgTQPO7IsVpv8WHcqO5yiTERE3cGwQy6kScXdXcICPFd2qk1miCKgEDoOMPTq3mI5RZmIiHzHsEMupKWinoQdT7N2pJ1YqXEaKBW+V43kk89Z2SEiIh8w7JCLMTlJiFErMblfSrffw1Nlp7PTzr0hTVFmzw4REfmi+//znSJSbkoMdi76NTQqZbffw2NlpwfNyYDT+Vis7BARkQ9Y2aEOehJ0AM+VnVN1TQB8n7EjkZex2LNDREQ+YNihXifN2altbEWr48gIURTx9c+nAcDjQaRdSeHWcyIi6gaGHep1STFquQFZqsJsLq5BUZUJsWolLhmV1e33BaRdXTwfi4iIvMOwQ71OoRDkKozUt/PBlhIAwGWj+3R7p5e0Ld5sscFktvbCnRIR0ZmAYYf8wrlvp9Zkxoo9ZQCA307M7fZ7xqhV0EbZ/5GtZZMyERF5iWGH/EIOO8YWfLLjBMxWG4b3iceI7IQevW8ypygTEZGPGHbIL6Tt55UNLfIS1rU9qOpIpB1Z7U8+t9pENLdyaYuIiDpi2CG/kCo7K/aextFKE2LUSlzWzcZkZ9JBpe23nz/21X4Mf+QbHKkw9vhnEBFRZGHYIb+QKjt7TxoAAJeNyoJOG9Xj902Osb+H8/Zzi9WGT7afgMUmYsfx2h7/DCIiiiwMO+QXae0GB/bGEhbgVNlx6tnZfaIeDS0WAEANZ/AQEVE7YRV2nnrqKQiCgIULF8qPNTc3Y8GCBUhJSUFcXByuuOIKlJeXB+8mCUDbYEEAGJoZj5E9bEyWSNvPnU8+X3+kSv4zd2kREVF7YRN2tm7din/+858YOXKky+N33303vvzyS3z88cdYs2YNTp06hcsvvzxId0kS5yMhrp2UC0Hw/ZRzd5LdTFFed7gt7PDcLCIiai8swo7RaMTcuXPxxhtvICkpSX68vr4eb775Jp5//nmcf/75GDduHJYtW4YNGzZg06ZNHt+vpaUFBoPB5Yt6V2ZCNHQaFRKiozBrdM8bkyVJ7baem1os2FHS1qfDsENERO2FRdhZsGABLr74YhQWFro8vn37drS2tro8PnjwYOTm5mLjxo0e32/x4sVISEiQv3Jycvx272eqWI0Kny2Yii/umIr4XmhMlsjLWI5Qs6W4BhZb29ER7NkhIqL2Qj7sfPjhh9ixYwcWL17c4bmysjKo1WokJia6PK7X61FWVubxPR966CHU19fLX6Wlpb192wRgQHoc+qbE9up7SstYUthZ61jCGpAeB4A9O0RE1FFIh53S0lLcddddeO+996DVanvtfTUaDeLj412+KDxIE5Qbmi0wW2xyc7I0w4fLWERE1F5Ih53t27ejoqICY8eOhUqlgkqlwpo1a/DSSy9BpVJBr9fDbDajrq7O5XXl5eXIyMgIzk2TXyVER8knqh8qb8DB8gYIAnDJyEwAgKHZglarLZi3SEREISakw84FF1yAPXv2YNeuXfLX+PHjMXfuXPnPUVFRWL16tfyagwcPoqSkBAUFBUG8c/IXhUJAkmOw4Be7TwEAhmcloG9KLBwZyGWnFhERkSrYN9AZnU6H4cOHuzwWGxuLlJQU+fGbb74Z99xzD5KTkxEfH48//vGPKCgowOTJk4NxyxQAybFqVBnN+NIRdqYOSIVSISAxRo0akxm1plak63pv2dPQ3AqzxSZPhSYiovAS0mHHGy+88AIUCgWuuOIKtLS0YPr06fjHP/4R7NsiP5K2n5+ubwYAnD0w1fF4FGpM5l7v25n9ynpUGluw+c8XIEYd9v/KEBGdccLuv9w//vijy/darRZLly7F0qVLg3NDFHApTtOZNSoFxvW1z15KidXgaKWpV8NOc6sVRVUmAMCpuiYMSNf12nsTEVFghHTPDpE70vZzAJiYnwxtlBIAkBRr7+XpzVk7zv0/NabWXntfIiIKHIYdCjvS9nPA3q8jPy4dJdGLlR3nKhG3tRMRhSeGHQo7zpWdaU5hR+rl6c1QUtfY6vRnhh0ionDEsENhJ9mxKyo5Vo2hmW0DIdtPV+4NLpUdhh0iorAUdg3KRGcPSMWwrHjMGdMHCkXbaeruTkTvKedqDo+iICIKTww7FHaSYtX4+s6z3T4O9HZlp9Xtn4mIKHxwGYsihtS43JsVGOcqESczExGFJ4YdihjSMla1yQxRFHvlPV23njPsEBGFI4YdihhS2Gmx2NDUau3y+rpGM55fdQgfbCnxeE0td2MREYU99uxQxIhRK6FWKWC22FBjMns82sFsseE/m47jxdWHUd/UCpVCwJXjshGl7Jj9azlnh4go7DHsUMQQBAHJMWqUGZpRa2pFdpLr86IoYtX+cixe8QuKHUdAAIDFJqLaaEZGQsfDQ52XsQzNFrRabW5DERERhS7+V5siijxrx82S07ubjmP+u9tRXGVCapwGT10+Amk6+8yeKmOL2/dr3+zsPGSQiIjCA8MORZS2wYIdw8s3+8oAAL8Zl40f7zsP10zMRboj7FQ2dLy+xWKFyWzv/YlS2uf5cEcWEVH4YdihiNI2a6djBeZ4dSMA4OoJOYjT2Fdw0zoJO1IVRyEA2Ukxjvdl2CEiCjcMOxRRkmPsJ5+3X35qsVhxqq4JAJCbEiM/nuo4eqLSzTKWVMVJilEjxRGiuCOLiCj8MOxQREmOtYeX9j07J2qbYBPtO7bSHAEH6LyyI1VxEmOiOq0YERFRaGPYoYiSHGuv7NQYXcPO8Wr77qu+KbEQhLbztNI6qexIy1jJsWokSRUjVnaIiMIOww5FlCQPu7GOVdn7dfKclrAAIFXajdVpZUftl3O3iIgoMBh2KKJ4Oh+rpMYedvqmxLo83nllxyy/pz/O3SIiosBg2KGIkhznCCXtKzvyMpZrZafznh37MlZibJTHihEREYU+hh2KKHIFprEVNlvbYaDStvMOYcdR2WlotqC53Xlabis7HCpIRBR2GHYooiQ6QonVJsLQbA8mFqsNpTVSz47rMlZ8tApqx/EP7aco1zhtPU+Kdb+lnYiIQh/DDkUUtUoBnWNgoNRMfKquGRabCLVKgYx41/OvBEHwuJQlVXGSYtVIYs8OEVHYYtihiNO+b+d4jb1fJzc5BgqF0OH6VMf1Ve22q0vBJikmSj6GoqHFArPF5p8bJyIiv2DYoYgjVWGqHeHlWLX7becSz5UdR9iJVSNeGwUpJ3GKMhFReGHYoYgjVWHkyk5V20BBd9yFnVarDQ3NFgD28KRQCHKI4o4sIqLwwrBDEUcOJY6t411VdqTzsZwblKXpyYIAJETbm5Ol7ee1PDKCiCisMOxQxJGOjJArO9W+V3ak1yZER0HpWL/ikRFEROGJYYcijnQYaLXRDJtNxPEa9zN2JO6mKEvNydJ8HcC5YsSwQ0QUThh2KOI4V3bKG5phttigUgjokxjt9nr5fCznsONYxkp0VHPs78vt50RE4YhhhyKOcwVGOgA0OykaKqX7f9zlyo6bZSwp4ACeDxklIqLQxrBDEcd5N1ZX/TpAW89Oo9kKU4tFfi3QNpEZ8HzIKBERhTaGHYo4UtipMZq73IkFALEaFaKjlADalrLknh03lR2ej0VEFF4YdijiOE87PlLRAADI7aSyA3TckeWuZ4e7sYiIwhPDDkUc52nHu0rrAXRe2QHchB13u7FiuRuLiCgcMexQxHGediwtS3XWswM4n48lVXbYs0NEFCkYdigiOffaCAKQk+x+27nE0zKWu54dk9mK5lZrr94vERH5D8MORaQkp5CSlRANjUrZ6fVpcVoAbYMF5UNAnXp24rUqeZpyHZuUiYjCBsMORSTnXhtPk5Odpers11c2mGG1iahvsoeZJJcKkcAmZSKiMMSwQxHJOaR01a8DuB4ZUd/UClG0P54YHeVyXRL7doiIwg7DDkWkFKew09VOLKCtZ6eqoUXebRWvVXWYuswpykRE4YdhhyKSr5WdVKfKjtyv4/QeEu7IIiIKPww7FJGkw0ABIC/V+8qO2WLDccfU5aSYjmGnbdaOa4NyWX0zfj5R193bJSIiP2LYoYjkHFRyk7sOO9ooJXRaFQDgcHmD4z2iOlznfKK6RBRF/O7NzZi9dD2KKo09um8iIup9DDsUkbIS7XN1+iRGI0at8uo1UpPyISnsuFnGkhuUncLO3pMGHK4wwiYCe07W9+i+iYio93n3W4AozJyl1+HpK0ZgQHqc169J1WlQVGXCoXJ7dcbtMlZMxyMjVuw9Lf+5uMrU3VsmIiI/YdihiHX1hFyfrpf6dk7WNQFwnZ4sSY51reyIoogVe8vk5xl2iIhCD5exiBykZSxJopueHWlpq9bRoHywvMEl4IRT2KlrNOPdjcdwpIJ9RkQU2VjZIXKQKjuSZDfLWMntlrFW7LFXdfqlxqKoyoSiShNEUYQgCH67z/qmVnz982lcMioT8dqOgcyb17+5rhj/XlcMY4sFU/qn4P1bJ/vhTomIQgPDDpFDx8qOu63n9nDR1Go/DFTq17n1nH7482d7YGyxoNLYgnSd1i/3aLOJ+MN/tmPD0WocrmjAI5cO8/q1Dc2tWLb+GN5YW4SGZov8+FHuICOiCMdlLCIH6XwsibuenTiNCirHYaDbjtXiULkRUUoBM4dnIjvJvgOsuNJ/S1n/WleEDUerAQAr95ZBlM618MItb2/D86sOoaHZgkF6HZ66fAQAoNzQwlPciSiiMewQOUgnn0vczdkRBEHu23l/y3EAwJT+qUiIiUJ+qn3nl7/6dvaerMez3xx03Adwur7Z663uoijK1z51+QisuOtsXD0hBzFq+2nwUlM2EVEkYtghcmjfs+NuGQto69v5dl85AGDmiAwA9r4dwD9hp8lsxcKPdqHVKuLCoXrMGJbhcg9daWixoNFsr95cNjoLCoUAQRCQk2QfuFhS09jr90xEFCoYdogcUuLawk2cRgW1yv2/HlLfjsUmQqkQ8OuhjrCTZg87RX4IO0/+7wCOVBiRrtPgqStGYroj7Hyzr6yLV9pVGJoBADqtymXIYk6yfentBMMOEUUwhh0ihyilQl66Sor1vMvJuZdncr9k+ft8P1V2Vh8ox7ub7Etmf79qFJJj1fjV4HSoFAIOVxi9OqKirL4FAKCPd12qy3EcpVFay2UsIopcDDtETqSlLHfTkyXOz80Ynin/WQo7x6tNsNq8bxzuTHOrFQ988jMA4JZp+Th7YBoAICE6CgX9UwAAq/Z3vZRV7qjsZLQPO45lrFJWdogogjHsEDlJjfM+7AgCMH2YXn48KyEaapUCrVYRJ3upUrKzpA5VRjPSdBrcN2OQy3MXDrX/bG+WssocYcdzZYdhh4giF8MOkZO2yo7nZaz0ePs1E/omu8zTUSgE5KdIfTu9M7tm27EaAMDkfinQqJQuz0m9QjtL6+SeHE8q5LDj2oQt9eyU1rgPZxarDY98vhefbD/h+80TEYWIkA47ixcvxoQJE6DT6ZCeno7Zs2fj4MGDLtc0NzdjwYIFSElJQVxcHK644gqUl3u3Q4WoPWmZJ7XdgEFnc8b0wY1T8/DEnOEdnuvtvp2tx2sBABPykjo8l5GgxaicRIgi8N2Bik7fR6rsZCS4X8aqb2qFobm1w+s2FlXj7Y3H8dfle9Fk5iweIgpPIR121qxZgwULFmDTpk1YtWoVWltbceGFF8JkavtFcvfdd+PLL7/Exx9/jDVr1uDUqVO4/PLLg3jXFM7mTuqLuZNycd3kvh6v0Wmj8MilwzBQr+vwXH5a74Udq03EDkfYGd832e013i5llRnsDcrtJzvHalRyg7W7vp1fTjcAsE+M/ulwpQ93T0QUOkL6uIiVK1e6fP/WW28hPT0d27dvxznnnIP6+nq8+eabeP/993H++ecDAJYtW4YhQ4Zg06ZNmDyZ5/2Qb3JTYvDEnBHdfr1U2SnqhSnKv5QZYGyxQKdRYVBGx2AFANOHZeDZbw5iw9EqNDS3QufhrKwKD5UdAMhJikaNyYzSmiYMy0podw8N8p+/2Vsmb3knIgonIV3Zaa++3j4BNjnZ/r9yt2/fjtbWVhQWFsrXDB48GLm5udi4caPH92lpaYHBYHD5IuoNnQ0WXLm3DAs/3InKhhav3mvbMXtVZ2zfJCgV7g8WHZAeh35psWi1ivjxoPvKi9UmoqJB2nrecXku29GkfMJNk/IvZW3/bnx3oBxmi82rew82Q3Mr1h+p8uk4DSKKXGETdmw2GxYuXIipU6di+HB7r0RZWRnUajUSExNdrtXr9Sgr81zWX7x4MRISEuSvnJwcf946nUGkys7JuiaX86YsVhse/nwvlu86hXv+uws2L7amb3U0J7vr13F24dDOBwxWm1pgtYlQCB0POwU8bz+3WG04XG5vtNZGKWBotmBTUXWX9x0KnlrxC+b+azO+2H0q2LdCRCEgbMLOggULsHfvXnz44Yc9fq+HHnoI9fX18ldpaWkv3CGRfeBgQrR9KelYdVt1Z82hSrmis/ZwFV5fW9Tp+4iiKIed8Xnu+3Uk0vb3Hw9Wuq28lDsGCqbGaaBSdvxXXt6R1W67fHGVCWarDbFqJeaM6QMAWLHXu4nNwSbtYlt3uCrId0JEoSAsws4dd9yBr776Cj/88AOys7PlxzMyMmA2m1FXV+dyfXl5OTIyPPcWaDQaxMfHu3wR9QZBENp2ZDn17Xy8zb51e0C6/bDQ5745iJ0ltR7f50RtE8oNLYhSChiVndjpzxyVnQidVgVji8Xt8pmnGTsST+djSf06gzJ0uMgxPHHV/rJeG5joToWh2WXprDtarTb5c9hVWtcLd0VE4S6kw44oirjjjjvw2Wef4fvvv0d+fr7L8+PGjUNUVBRWr14tP3bw4EGUlJSgoKAg0LdLBKCtb0c6I6vGZMbqX+zjEF757RhcPCITFpuIOz/c6Xa7NwBsO26vTAzvk4BotdLtNRKFQkC/NOnE9Y7zfcq7CjtOPTvOPS5S6BiUEY/J/VIQr1WhymjG9uOeQ1pP3fLONlz68jqvjsDw5FiVCa1W+9/jSKURDR4+YyI6c4R02FmwYAH+85//4P3334dOp0NZWRnKysrQ1GQvtyckJODmm2/GPffcgx9++AHbt2/HjTfeiIKCAu7EoqBpP2tn+c6TaLWKGNEnAYMz4vHk5SPQJzEapTVN+Mtne9020W49Js3X6XwJS9I+YDkr9zBQUNInMRqCADS32lBpbGuelradD8nUQa1SoHCIfblspZ+WshrNFuw5WY9Wq4i1PVh+OlzRFpREEdhzor43bo+IwlhIh51XX30V9fX1OO+885CZmSl/ffTRR/I1L7zwAi655BJcccUVOOecc5CRkYFPP/00iHdNZ7r2s3Y+dkwf/s14+xJsQnQUXrp2DJQKAV/uPiUvcTmTek7G9+28OVn+mW6WziSezsWSqFUKZDqec56kLC1jDc6wL/NOH97WCO2PXU5FlSZIb7vF8ffvjkPlDS7f7zpR14O7IqJIENJhRxRFt1833HCDfI1Wq8XSpUtRU1MDk8mETz/9tNN+HSJ/c67s7D1ZjwOnDVArFbhsVJZ8zbi+Sbjn12cBAP725T6XnVB1jWYccuyCGudr2HHbs+P+xHNn7befG5pbcbLOHnwGOYYnnntWGqKjlDhZ14S9J3t/XMPhiraQsu1YTbcDlbSDTAp3u9m3Q3TGC+mwQxSO8hznY9WYzHhzXTEA4NfD9Ehsd7joH87tjwl5STCZrXjw05/lX+5ST0z/tFikdHJshbPOwo58LpabgYKS9tvPDzqqOlkJWiQ4zgnTRinxq8H2U9dX7jvt1X35QgopAFBuaPF4XldXpMrOFePsO8jYpExEDDtEvSxWo5KrCst3nQQA/GZcdofrFAoBz1w5ChqVAuuPVOODLfYRCL726wBtYafaZEZ9o2tDblkXy1hAxwNB5SWsTNeditIE5RV7e38py7nXBujeUpbZ0rYTa86YbCgVAsoNLSir7/ygVCKKbAw7RH4ghQ9RtIeMswemebzuvumDAABP/u8ATtY1tfXr+BB2YjUquQG52Gm+T3OrFXWO8OOpQRlwquw4lrF+OS3txHI9puL8welQKxUoqjThYLveGGdHKhrw8PK9+P2723DN6xsxY8lPmLJ4NW55e5vHretHHGFnRB/7kRVbi30PO8eqTbDYRMRpVOifFouzHEtwu0r9t4OMiEIfww6RH0hNygBw+dg+Ho97AIAbp+ZjXN8kGFssuO/j3fjZsXuoq8nJHX6mvJTVViGpcPTraFQKedihO9L2cznsyM3JrmFHp43C2QNTAQDz/r1FHnzo7PNdJ3HZK+vx7qbj+GZfOTYV1eCXsgacqm/GdwfKceB0x36f5lYrjjtC2m8n5QKA2/fuirSENSA9DoIgYHSOPTjtKuWOLKIzGcMOkR9IW8EB4Eo3S1jOlAoBz1w5EhqVAhuOVsNstSFNp0GuI4B4Kz/VMWvHaUeW80BBQfAcuKRlrFN1zWi12uSenSGZHQduLrp0KPqnxaLc0IJrXt+Ef645ClEUYbbY8Mjne3HXh7vQaLaioF8KHps9HC9dOwbv3DQRY3MTAQC73eyOKq4ywSYC8VoVLnLs+iqqMqHK6N05YhKpsfssvf2zGJ3j+Jm92Lfz3DcH8bs3N7scB0JEoY1hh8gPxjp2UU0bkCoP/OtM/7Q4/OnCs+TvJ+QldRpO3HE3a6erbecSvU4LtVIBq03EtmO1MLZYoFYq5GqRs74psfjijmmYNToLVpuIxSt+wa3vbMfVr2/E2xuPAwDu+NUA/OeWSfjd5L64bFQWzjkrDVP62ytCu0rqOryntIQ1ID0OiTFqeQfYNh+rO4cdlR1p+WqUI+z8fKKuVyY/t1is+OdPR7H2cBU2d2OZjYiCg2GHyA/G5ibhyzum4R/XjfX6NTdP64cxjuqHFAx84W5HlhR20jvp1wHszdJ9kuzVnVX77dOe+6fHIcrNWVqAvUdoydWj8fjs4VArFfjuQDl2ltQhXqvCm/PG497pgzos3UnBw11lR2pOHphuDykT8u1hcUuxb7020jLWQEfYGZiuQ4xaCZPZiqM9mMosOVjWIE9n3n+q97ffE5F/MOwQ+cmI7ATEaz33ybSnVAh464aJePnaMbhmQo7PP895mKG0U0rahdRVZQcAsqWwc8A+IXlIu36d9gRBwHWT++KTP0zBWfo4jO+bhK/vPBsXOCYttzfK0T9zuKLjEQ5HKqSQYq+CSTvRfOnbabFYcaza3nMkLWMpFYLc8OyuouSrn52mMe87xT4gonDBsEMUQhJionDpqCy3p5N3JScpBkqFgEazFRWOE9bLHf83o5MZO/LrpSZlx/bzwZmdhx3JiOwEfHv3ufh/f5giv4c76Tot+iRG249wOOkaFKQZO9JBqRPz7WFn36l6GFssXt1HcZUJVpsIndPWf6Ctb6c3Jin/7PQe+900WhNRaGLYIYoQapUCOY7qTJGjSbm8XlrG6jrstG+IHpTRsTm5p0bJu6Pq5MecTymXlp8yE6KRnRQNm4hOT4d3JjUnD9THufQ79WaTsnNlp7jKhEazd0GMiIKLYYcogrTv2ylv8H4ZS5q1I+lqGas73AWP447ZOLFqJbKcKlATpaUsLxuB2zcnS6ReoV/KGtBk7v4OqiazVe4Jio5SQhTbtugTUWhj2CGKINLOr+IqI0RRlHt2OhsoKJG2nwNAcqwaaTrvjqrwxajsRACulR3nJSzniow0VNHbScrtm5MlmQlapOs0sNrEHvXZ7DtVD5sIpOs08jIbm5SJwgPDDlEEca7sGJosaLHYAHR+CKjEubIzSK/zeeu7N0ZkJ0AhwOUIh8PytnPXkDLRsSNrZ0kdzI6/R2cOt5uxIxEEQa7u9OScLGkJa2R2IoZm2Zf4etK302S2+jxHiIi6RxXsGyCi3uM8a0caKJgQHQVtlLLL1ybGRCFOo4KxxeJ1c7KvYtQqnKXX4ZeyBuwqrcOMhAx5xs7AdiGlf1ockmPVqDGZsedkfacnwDe3WnHMMYG5/TIWYF8+W7W/vIdhx/7akdkJcqj0pbKz50Q9NhVVY++peuw7ZUBRpRE2EXj35okejxMhot7Byg5RBJG2n5dUN+JknX0btjf9OoC9AiJtP29/TERvkmYJScGjbcZOx4rMeEfA6WoLelFl2wTmdDfLb1Kv0M4ebD//+aRU2UmQKzu/lBm8GlZ4sKwBl76yDk/87wA+33UKRyrsQQcAvtx9qtv3RETeYdghiiB6nRbRUUpYbCK2H7fvYtJ7se1c8vtz++Hsgany6eb+IPXt7C61TzWWhv0NTO8YsKTemBV7y2Cxel7KOlzR1pzsbvltZHYCopQCTtY1yTN9fGFobpV3uI3MTkReSiyio5RobrW5DHH05LsD9kGN/VJjce+FZ2HZDRPwwtWjAADrDlf1+gnyROSKYYcogigUAvIcSywbj1YDAPQ+NBrPGZONd2+ehMQYtV/uDwBGOyo7P5+ow7FqE8wWGzQqhTzB2dmM4RmIVSuxu7QOz317yON7empOlui0UZg6wD6VesWeMp/vea+jXyc7KRrJsWooFYK81OdN3866w1UAgBum5uGO8wfiV4PTMX1YBtRKBU7VN3sVmDxpaG7F3789iBLHQEUi6ohhhyjCSH07UkOtNwMFA8n5CIeVe+3Bo39anNuT4bOTYvDMlfYKyGtrjsrXt9f+AFB3pANGV3h4j844L2FJhjoOSe2qb6fJbJWrbNMGtB0DEqNWYWzfRADA+iNVPt+T5I2fivDy90fw7LcHu/0eRJGOYYcowkjNsxZHU4g3AwUDyfkIh0+2nwDQsTnZ2cUjM3HLtHwAwL0f70aRmzOuPM3YcfbroRlQKgTsP23wuQrS1pycKD8m9e10tZ19c7H9JPs+idEdDlaVws/aw90POxscFby9J3l8BZEnDDtEEab9L1RvG5QDSWoYlk5ob9+c3N4DFw3GxLxkGFssuO0/210mFzc0t6Kkxh5eOgtNybFqTJJ7gE77dL/ytvM+bZWdYVn2P+8/Zei050Zawpo2ILVDP9E0xy6sjUXVnfYkedJotsgHqxZXmbw+WuNMs3LvaYz8v2+w4Wj3QyWFN4Ydoggj7ciSeDNQMNCkuTeS9jN22otSKvDK3DFI12lwqNyI2/6zAw/8v59x0YtrMfrRVbCJ9q3zaXGd/127s5RVbWzBiVr7eWHDnZaxBul1UAhAtcmMygbP83LWOZaopg3seJL9iD4JiNeq0NBskZfKfLH9eK18CjsAHAjieV2iKMLQ3IrSmkbsOVGP6hCaIfTpjpMwNFvwxS7ufDtTcc4OUYTpF0aVHUlnFRlJuk6LpXPH4trXN+GnQ5Uuz6XEqnHbuf27HIQ4fVgGFn2xD7tK63C6vgmZCR2botuTDi3tlxbrcop9tFqJfmlxOFJhxL7TBrfLhRUNzfilrAGCALlB2plSIWBK/1Ss3FeG9YerMDbX8ywhd6QmdMm+k/XyifGB8sPBCvz50z2oaGhx2YafEqvGugfOR7S66xlP/iaNNzjA4z3OWKzsEEWYxBg1kmLsv5SVCgEpXVQ7giEzQSsfRxGlFNC3k9PSnU3IS8bfrxqFwiF6LPhVf7x23ThsePB8bPtrIW49p1+Xr0+P12KcI1C4a3a22cQOS1LulrAkXTUpS43Hw7LikRzrfofbVEfFZ103mpQ3FdnDjjQfaV8Qjq94fU0RTtc3y0FHG6WAUiGg2mTGzlLvDnH1p+ZWK447Bk4eKmuAzYu5SBR5GHaIIpDUt5MWp3G7yynYBEGQqzv9UuOgUnr/n6JZo/vgX/PG477pgzFjeAayEqN9OtpihoelrBqTGZe8vA7Tnv4BG5yCh7vmZElXx0aslft1PE9IPttR8dlRUguTDz03phaLHMRumJIHIPBhp9Zkls8u++KOqfjlsRn45bGLMHNEJgBg27Hghx3nAY5NrVa5v4vOLAw7RBEoP9W+LOTLQMFAk6YjD/HT0RSeSGFn67EaudemyWzFzW9vxf7TBpysa8LcNzfj2W9+QavV5nQmlufKzgE3IUMURbk5+Ww3/TqSvikx6JMYjVar6PWhpwCw7XgtLDYRfRKj5SGQhysavDpHrLes/qUCVpuIwRk6jMxOlI8lmZDn3eTrQDjcbojkL2U8vPVMxLBDFIH6OZqUM0KwOVkyb0oeHrxoMO6bMTigPzc7KQYjsxMgisC3+8tgtYm488Od2FlSh4ToKMwZ0weiCCz94ShmL12PioYWKIS23VfOhjjCTnG1qUNV5nCFERUNLdCoFJ2e6yUIgrwFfZ0PW9Clfp2C/inITopGvFaFVqvY4Ze7P32zz14daz9xW+ob2nG8tlu7zHqTNINJcuA0+3bORAw7RBHo8rF9cNHwDNw8res+lmDRRilx27n90Sex6ybh3iZVd1buLcP/fbEPq/aXQ61S4F/zxuOFq0fjld+OgU6rkpeFztLr3Dbapuk0SNdpIIrAL+2aX6UlrIn5yV0exCrt1PJluKDUrzO5XwoEQXCa++Nd5eKl1Ydx6cvrOt1J1pkmsxVrD9sbxS8cpnd57iy9DjqtCiaztcPnEmjSDKa8FHtfGCs7ZyaGHaIIlJkQjVevGyefLUWuLhpu7ylZe7gK7246DkEAXrx6tFyRuGRkFv5359kY6zjaYnK/FI/v5alvZ50jCHS2hCWZ0t/+/r+UNaCiobnL640tFnmX2OR+9nt2nvvTla3HavD8qkPYc7IeX/3cve3Yaw5VornVhuykaHk5T6JUCHI1a0txcJeypMrOpaOyANgPZaUzD8MOEZ1x8lNjXU52f/jiobjI0VQryUmOwUe/L8AHt07G/TMGeXwv6Rf9usOVcr+M2WLDZscv+c6akyUpcRoMc4SmDUequ7jaHlasNhE5ydHITrJXLIZ5OdHZbLHhz5/ukb/v7lEV3+63L2FdODTDbYO4FBy3He9+2Pl810nc/dEuGJpbu/X6JrMVpbX2huTLHGHneE2jT43gFBkYdojojHT1hBwAwO/P6YebHMdRtBelVKCgfwpi1J5HkknVs2/2leNXz/2I9zeXYHNxNRrNVqTGqV1CVWfkvh0vwscmqV/HqeLkXNnpbHv1P9ccxeEKI7RR9v/8byqqQauPfTWtVhtWH6gAAExvt4QlkcLOluLabp3qbmqx4K+f7cVnO09i6Q9HfH49YN+JJYr2mT8D9TqkOZYcpYNju/Lol/tx9T83oq7R3K2fT6GDYYeIzkg3TMnD1r8U4qGZQ3r0PucNSsdjs4cjXafBybom/PmzPbhx2VYAwJT+qVB4ufVf6ttZe7jSZTifO879OpL+abFQqxQwma047mF7dXGVCS87gsPiy0cgMSYKxhaLvL3eW1uLa1Df1IrkWDXGexhiODI7AWqlAlXGFhx3cxbZ4hUHcOOyLR6rLF/sPoUGx3NvbziGCkPXy3vtHXSEGmlopRQ8vekjqjWZsWxDMTYX1+DRr/b7/LMptDDsENEZSRAEebBhT/1ucl/8dP+v8MilQ5Gu08iHsHrTryOZkJeMxJgolBta8P0vFR6vMzS3OvXrtIUdlVIh/zJ317cjiiL+8tkemC02nD0wFbNH98HU/tIusK6XzpxJu7AKh6R7nOOkjVLK2/Xbb6k/cNqAf64pwg8HK/HOxuNu7/U/m+yPq1UKNLfa8Eo3qjvtD4iVw44Xx2qsO1IFqSD16Y6TWH2g3OefT6GDYYeIqBdoo5S4cWo+frr/V/jbZcNw27n9MWt0H59eLy2tvb3hmMfrth2rgU20z+fJareTrbO+nU93nMSGo9XQqBR4fPZwCIIgH2HhS9+OKIr4dr/9F/+FQzM6vVaq+mxrF3be+Kmo7c9ri1wOdgWAXaV12HfKALVKgSVXjwYAfLClBKU+DgQ81CHs2D8fbyo70k6zRMc08j9/tgf1jd3rHaLgY9ghIupF2iilPENIrfLtP7G/m9wXCsFeVTjiYV7ORjf9OpKhjr6d9tvPa0xmPP61fSnmrsKB6Jtin8M0rRvTm/ecrMfp+mbEqJVuDzd1NjHfviPLeZLyqbomfLHbvgMsOVaNGpNZruJI/rOpBABwychMzByRiakDUtBqFfHS6sNe3aNE2oklh53MtmWszvqIRFGURwc8e+Uo9EuNRbmhBY99zeWscMWwQ0QUIrKTYlA4xN7w+/aGjss7gL2hGHC/HX6Yh1k7j365D7WNrRik1+HWs9tmL+WmxCAnORoWm+j1FvFv99mrOucNSutyftC43GQIAlBUZZLn+by14RgsNhGT+yXjwYvsAyVf/6kITWYrAHuvjLQd/rrJfQEA915o3w33yY4TOFJhbP9j3DK1WHCyzn5a/VmOnp0B6XFQKgTUN7WirJMeoCMVRpyub4ZGpcDZA1PxzJUjIQjA/9t+Aj8c9LzEGAzbjtXg1R+P4tMdJ7DxaDWOVZnQ3GoN9m2FHIYdIqIQIp1z9cmOEx22XP/wS4Xbfh3JkIx4KASgytgiN/R+t78cy3edgkIAnr5yJKLanUPmyy4woK1fp6slLABIiInCIEdVZfvxGhiaW/H+ZnvV5vfn9MecMX2QkxyNKqMZ7222h7v/t/0EWiw2DMuKxxjH+WljcpNQOEQPmwi88N0h+f2PVBjx5P8O4NZ3tqG8XXiRTjpP02mQGGM/hFWjUqKf49y4XzqZpPyTo6ozqV8KtFFKjM9Lxo1T7Dv2HvpkT7e3wve2WpMZ1/97C55e+Qvu+e9uXPvGJpz33I8YumglXltzNNi3F1IYdoiIQkhB/xScpY9Do9mKj7edkB8/Xd+Ee/67CwBwfUFfZLg59yxarZQPgd13yoD6plb8Zbl9ps6tZ/eTD1915m3fTqvVhv/7Yh8OVxihUgj41eB0r/4+4/Ok4YK1+GBzCYwtFpylj8N5g9IQpVRgwXkDAAD/dFR3pNBz3eS+LvN7/nThWRAE4OufT2PpD0dwxasbUPj8Grz+UxFW7S/Hy9+7LnEdKpP6deJcHh+c2XXfzk+H7P065zgt0903fRD6psSgzNCM297d7tXwR397e+MxNJqtyEzQYuqAFPRLjYU2SgGbCDyz8hfsPdn5zKUzCcMOEVEIEQQB8xzVnXc3HoPNJsJiteHOD3aitrEVw7Li8edOtsvL83ZOG/DE1/tRbmhBv9RY3P3rs9xeP8WxI6uz6c21JjPm/XsL3nI0Tj940WAkREd59feR5u1sOFqFZevtr7/17H5ykLl8bDb6JEajsqEFd364E8eqG6HTqDBrdJbL+wzJjMelI+2PPfvNQWw/XgulQsBEx/t/uuOkS8VFak4emO4656ht+7n7HVnNrVZsLrb3RZ09sG0gZLRaib//ZhQ0KgU2HK3GRUvWBnWHVqPZIv//46GZQ/DeLZPx/b3n4cCjM3DJyEzYROAvn+3pcozBmYJhh4goxMwZ0wfxWhWOVTdizaFKPL/qELYeq0WcRoWlvx3baa+M1Lfz0dZS/HfbCQgC8MyVIz2+JjlWLb9Gan52dqi8AbP/sR4bjlYjRq3EP383Drc49f10RQo7v5Q1oMzQDH28xmWXmlqlwIJf2as7qxy7vC4f28ftIMc/XXgWUuM0yE+Nxf0zBmHDg+fjo99Plithn2xvq4QdqnBtTpYMkZqUPSxjbTtWi+ZWG/Txmg5VofF5yfjyj9MwOEOHapMZN7+9DYs+3xuUHpkPtpSirrEVfVNiMHN425KiIAhYdMlQ6DQq7D5Rj/c3u+/9OtMw7BARhZgYtQpXjbdvQ//bl/vwjx/t/ReLLx+BPMcylSdSZafEsU17XkGex8F/Ek+nrq8+UI7L/7EBx6sbkZMcjU9vn9LhhPOuZCVGuxz2euPU/A671K4cl+1yzVxHY3J7fVNise2vhfjh3vNw+3kDoI/XQhAEXF+QBwB4Z+NxeXp024yddstYju3nRyuN8vEeztbKZ5qluT0G4yy9DssXTMXNjqnb72w8jpkvrsW/1hahrD4wS1tmiw3/Wmvfvv/7c/pD1a4PKz1ei/scR5w8s/JgSCy5BRvDDhFRCLq+IA+CABxzTB/+7aRc+TDLzkgHkwJATnJ0p+d6SZz7dqQt2cvWF+PWd7bB2GLB5H7J+HzBNDko+Erq24nTqPDbSbkdnneu7kzpn9KhGtOVOWP6QKdRobjKhLVHqmBobsVpR/AY2O69MhO00GlVsNhEHK3suLNLak7ubCCkNkqJhy8Zindumog0nQZFVSY8/vUBFDy1Gte8vhHvby5BQxdNzKU1jagydu/E+eW7TuJ0fTPSdBpcPtb9LKe5k/piVHYCGloseOyrA936OZGEYYeIKATlpsTgAkcT8OAMHRZdMtSr1yXHqjFIr4MgAE9dPrLTc70kE/KSoVYqcKq+GUcrjfi/L/bhb1/uh00Erp2Yg3dvnoTkWHW3/y5Sr83vz+mHeK37Xp9rJ+bgrRsn4OVrx/j8/rEaFa4cnw0AeGfDMRx2zNfJiNd26C0SBAFD5OGCrn07FQ3NOHDaAEFw7dfx5Jyz0vDd3efisVnDML5vEkTRPhrgz5/twUUvrnV7wrooinjjpyKc8+wPmL10vc9LYDabKO+0umVavsflSaVCwBNzRkAhAF/uPiU3XZ+puv63gIiIguKRS4chLyUWN3byS82dd26eiNpGs9eVmGi1EuP6JmFjUTXm/mszyg32isNDFw3G/HP6uV3O8UXhUD12Lfp1p03NgiDgvEHe7fBy53eT+2LZ+mP4/mAFznI0IQ9st4QlGZypw5ZjNfa+HadsJS3jDc9K8DrcJcRE4XcFefhdQR5O1Dbiq59P492Nx3GitgmX/2M9Xv7tGJw/2D47qdVqwyNf7JO335+otQ9YlJYsvfHt/jIUVZoQr3VfJXM2vE8CbpiSj3+vL8a9H+/GoAwdGpotMDS3oslsxUXDM/HnmYM7LIP1VHOr1ad/XgOBlR0iohCVkxyDv14y1KWfxRv6eK3PS07SNORyQws0KgVenTsWvz+3f4+DjiQxRt1r7+VOv7Q4nHtWGkQReHNtMYCOzckS6bM50K7yIm85P8v7M82cZSfF4LZz++OrP07D5H7JMJmtuPntbfjX2iIYmltx01tb8f7mEggCMLmfvY/qzbXFXp8KL4oiXnX0b11fkAedhyqZs3suPAsZ8VpUNLRg7eEq7CqtQ1GlCafrm/Hv9cX44wc70WLpnQZrU4sFD336M4YuWomHl++V+6dCASs7RESEC4ak47lvDyIlVo03rh+PMblJwb4ln82b0hdrDlXCbLU3HrdvTpYMclR+9p2sx46SWozskwCFIMiDFb1ZwupMUqwa7948CYs+34sPtpTi8a8P4JUfjqCusRUxaiVeumYMJuQnY8ri1ThY3oC1h6twzlnuf6YoijhR24TNxTVYd7gSu0/UQxulwI1T87y6lziNCu/fOgnrjlQhVq1CfHQUdFoVTtQ24c+f7sGKvWUwvbMd/7xuHKLV3a/GbD9eg7s/2i03xr+76TiMLRY8e+XIXq8cdQfDDhERYXBGPFbcdTb0Oi2SetCfE0znnpWO3OQY+Rdu++ZkyaAMHaKUAqpNZlz+jw3QaVQYmZOAKqMZsWolxvZC0ItSKvDknBEYmK7D41/vR11jK/TxGrw5bwKG97HvmLtqQg6WrT+GN9YWdQg7NpuI5749iM92npSbrSU3Ts1HSpzG63vplxaHfmkdg19GvBa3vrMNPx2qxPX/3ow3b5jgsafKE7PFhpdWH8Y/fjwCmwj0SYzGVeNz8PL3h/HZzpNoMlvx0rVjfD4nrrcJorf1swhmMBiQkJCA+vp6xMd3b7cBEREF37/WFuHxr+27j/b834Uel3pW7S/HJ9tPYMPRKhia2w5BvWBwOt68YUKv3tOGI1X47kAFbj0nH5kJbUuSpTWNOPfZH2ATgW8WniNXnABgyXeHsOQ7+1ToKKWAkdmJmJifjMn9UnDOwNReWxLcfrwGNyzbioZmC4b3icdL14xxG4zc2XuyHg988rN8FtvlY/rg/2YNQ7w2Cqv2l2PBeztgttpw3qA0vHbdOL/08Xj7+5thBww7RESRor6pFXP+sR790+LwxvXju7zeahOx71Q91h2pwuFyI+af0w9DMgP3e+D297bjf3vKcNX4bDxz5SgAwMq9ZbjtP9sBAIsuGYprJ+b2aImpK/tO1eP6N7eg2mSGSiFg7qRc3HnBQI/Vo0azBS+sOoQ31xXDJgKJMVF4YvYIXDwy0+W6nw5VYv6729DcakNBvxT8a954xGp6d0GJYccHDDtERJFDFEW/NkP3pu3Ha3HFqxugViqw7sFfodbUisv/sR4msxU3TMnD/102LCD3UVLdiEe/2ofvDthPdddpVLjtvP64fGwfqBQKqBQCFAoBO0pq8fDyvThRaz9R/pKRmVh06VCk6zqe1QYAW4prcNNbW2EyW/DmvPHyzrTewrDjA4YdIiIKlsv/sR47SupwfUFf/HiwEiU1jSjol4J3bp7Y4ZR6f9twtApP/u8A9p50f3aYpE9iNB6bPcyr8LKrtA6Hyht82mLvLYYdHzDsEBFRsPxvz2nc/t4O+fvspGh8cce0Hg1y7AmbTcTnu0/ixe8Oo7S2yeUwUbVSgesm98WfLjyr15ekusPb39/Bv1MiIqIz2IVD9chOisaJ2iZERynxxvXjgxZ0AEChEDBnTDbmjLFPpRZFETbR3t8kCAh4tak3hN8dExERRRCVUoGHLhqCPonRePGa0QFtkPaGIAhQKgSoVYqwDDoAKztERERBd/HIzA67maj3hGdEIyIiIvISww4RERFFNIYdIiIiimgMO0RERBTRGHaIiIgookVM2Fm6dCny8vKg1WoxadIkbNmyJdi3RERERCEgIsLORx99hHvuuQePPPIIduzYgVGjRmH69OmoqKgI9q0RERFRkEVE2Hn++edx66234sYbb8TQoUPx2muvISYmBv/+97+DfWtEREQUZGEfdsxmM7Zv347CwkL5MYVCgcLCQmzcuNHta1paWmAwGFy+iIiIKDKFfdipqqqC1WqFXu968qper0dZWZnb1yxevBgJCQnyV05O75/ESkRERKEh7MNOdzz00EOor6+Xv0pLS4N9S0REROQnYX82VmpqKpRKJcrLy10eLy8vR0ZGhtvXaDQaaDSaQNweERERBVnYV3bUajXGjRuH1atXy4/ZbDasXr0aBQUFQbwzIiIiCgVhX9kBgHvuuQfz5s3D+PHjMXHiRCxZsgQmkwk33nhjsG+NiIiIgiwiws7VV1+NyspKLFq0CGVlZRg9ejRWrlzZoWnZE1EUAYC7soiIiMKI9Htb+j3uiSB2dcUZ4MSJE9yRRUREFKZKS0uRnZ3t8XmGHdh7fE6dOgWdTgdBEHrtfQ0GA3JyclBaWor4+Phee1/qiJ914PCzDhx+1oHFzztweuuzFkURDQ0NyMrKgkLhuQ05IpaxekqhUHSaCHsqPj6e/+IECD/rwOFnHTj8rAOLn3fg9MZnnZCQ0OU1Yb8bi4iIiKgzDDtEREQU0Rh2/Eij0eCRRx7hAMMA4GcdOPysA4efdWDx8w6cQH/WbFAmIiKiiMbKDhEREUU0hh0iIiKKaAw7REREFNEYdoiIiCiiMez40dKlS5GXlwetVotJkyZhy5Ytwb6lsLd48WJMmDABOp0O6enpmD17Ng4ePOhyTXNzMxYsWICUlBTExcXhiiuuQHl5eZDuODI89dRTEAQBCxculB/j59y7Tp48ieuuuw4pKSmIjo7GiBEjsG3bNvl5URSxaNEiZGZmIjo6GoWFhTh8+HAQ7zg8Wa1WPPzww8jPz0d0dDT69++Pxx57zOVsJX7W3fPTTz/h0ksvRVZWFgRBwPLly12e9+Zzrampwdy5cxEfH4/ExETcfPPNMBqNPb85kfziww8/FNVqtfjvf/9b3Ldvn3jrrbeKiYmJYnl5ebBvLaxNnz5dXLZsmbh3715x165d4syZM8Xc3FzRaDTK19x2221iTk6OuHr1anHbtm3i5MmTxSlTpgTxrsPbli1bxLy8PHHkyJHiXXfdJT/Oz7n31NTUiH379hVvuOEGcfPmzWJRUZH4zTffiEeOHJGveeqpp8SEhARx+fLl4u7du8XLLrtMzM/PF5uamoJ45+HniSeeEFNSUsSvvvpKLC4uFj/++GMxLi5OfPHFF+Vr+Fl3z//+9z/xL3/5i/jpp5+KAMTPPvvM5XlvPtcZM2aIo0aNEjdt2iSuXbtWHDBggHjttdf2+N4Ydvxk4sSJ4oIFC+TvrVarmJWVJS5evDiIdxV5KioqRADimjVrRFEUxbq6OjEqKkr8+OOP5WsOHDggAhA3btwYrNsMWw0NDeLAgQPFVatWieeee64cdvg5964HHnhAnDZtmsfnbTabmJGRIT777LPyY3V1daJGoxE/+OCDQNxixLj44ovFm266yeWxyy+/XJw7d64oivyse0v7sOPN57p//34RgLh161b5mhUrVoiCIIgnT57s0f1wGcsPzGYztm/fjsLCQvkxhUKBwsJCbNy4MYh3Fnnq6+sBAMnJyQCA7du3o7W11eWzHzx4MHJzc/nZd8OCBQtw8cUXu3yeAD/n3vbFF19g/Pjx+M1vfoP09HSMGTMGb7zxhvx8cXExysrKXD7vhIQETJo0iZ+3j6ZMmYLVq1fj0KFDAIDdu3dj3bp1uOiiiwDws/YXbz7XjRs3IjExEePHj5evKSwshEKhwObNm3v083kQqB9UVVXBarVCr9e7PK7X6/HLL78E6a4ij81mw8KFCzF16lQMHz4cAFBWVga1Wo3ExESXa/V6PcrKyoJwl+Hrww8/xI4dO7B169YOz/Fz7l1FRUV49dVXcc899+DPf/4ztm7dijvvvBNqtRrz5s2TP1N3/03h5+2bBx98EAaDAYMHD4ZSqYTVasUTTzyBuXPnAgA/az/x5nMtKytDenq6y/MqlQrJyck9/uwZdihsLViwAHv37sW6deuCfSsRp7S0FHfddRdWrVoFrVYb7NuJeDabDePHj8eTTz4JABgzZgz27t2L1157DfPmzQvy3UWW//73v3jvvffw/vvvY9iwYdi1axcWLlyIrKwsftYRjMtYfpCamgqlUtlhZ0p5eTkyMjKCdFeR5Y477sBXX32FH374AdnZ2fLjGRkZMJvNqKurc7men71vtm/fjoqKCowdOxYqlQoqlQpr1qzBSy+9BJVKBb1ez8+5F2VmZmLo0KEujw0ZMgQlJSUAIH+m/G9Kz91333148MEHcc0112DEiBH43e9+h7vvvhuLFy8GwM/aX7z5XDMyMlBRUeHyvMViQU1NTY8/e4YdP1Cr1Rg3bhxWr14tP2az2bB69WoUFBQE8c7CnyiKuOOOO/DZZ5/h+++/R35+vsvz48aNQ1RUlMtnf/DgQZSUlPCz98EFF1yAPXv2YNeuXfLX+PHjMXfuXPnP/Jx7z9SpUzuMUDh06BD69u0LAMjPz0dGRobL520wGLB582Z+3j5qbGyEQuH6q0+pVMJmswHgZ+0v3nyuBQUFqKurw/bt2+Vrvv/+e9hsNkyaNKlnN9Cj9mby6MMPPxQ1Go341ltvifv37xfnz58vJiYmimVlZcG+tbD2hz/8QUxISBB//PFH8fTp0/JXY2OjfM1tt90m5ubmit9//724bds2saCgQCwoKAjiXUcG591YosjPuTdt2bJFVKlU4hNPPCEePnxYfO+998SYmBjxP//5j3zNU089JSYmJoqff/65+PPPP4uzZs3iduhumDdvntinTx956/mnn34qpqamivfff798DT/r7mloaBB37twp7ty5UwQgPv/88+LOnTvF48ePi6Lo3ec6Y8YMccyYMeLmzZvFdevWiQMHDuTW81D38ssvi7m5uaJarRYnTpwobtq0Kdi3FPYAuP1atmyZfE1TU5N4++23i0lJSWJMTIw4Z84c8fTp08G76QjRPuzwc+5dX375pTh8+HBRo9GIgwcPFl9//XWX5202m/jwww+Ler1e1Gg04gUXXCAePHgwSHcbvgwGg3jXXXeJubm5olarFfv16yf+5S9/EVtaWuRr+Fl3zw8//OD2v8/z5s0TRdG7z7W6ulq89tprxbi4ODE+Pl688cYbxYaGhh7fmyCKTmMjiYiIiCIMe3aIiIgoojHsEBERUURj2CEiIqKIxrBDREREEY1hh4iIiCIaww4RERFFNIYdIiIiimgMO0RERBTRGHaIKGDy8vKwZMkSr6//8ccfIQhChwNHI5Wvnw8ReUcV7BsgotB13nnnYfTo0b32C3jr1q2IjY31+vopU6bg9OnTSEhI6JWfT0RnJoYdIuoRURRhtVqhUnX9n5O0tDSf3lutViMjI6O7t0ZEBIDLWETkwQ033IA1a9bgxRdfhCAIEAQBx44dk5eWVqxYgXHjxkGj0WDdunU4evQoZs2aBb1ej7i4OEyYMAHfffedy3u2X6YRBAH/+te/MGfOHMTExGDgwIH44osv5OfbL2O99dZbSExMxDfffIMhQ4YgLi4OM2bMwOnTp+XXWCwW3HnnnUhMTERKSgoeeOABzJs3D7Nnz+7077tu3TqcffbZiI6ORk5ODu68806YTCaXe3/sscdw7bXXIjY2Fn369MHSpUtd3qOkpASzZs1CXFwc4uPjcdVVV6G8vNzlmi+//BITJkyAVqtFamoq5syZ4/J8Y2MjbrrpJuh0OuTm5uL111/v9L6JqGsMO0Tk1osvvoiCggLceuutOH36NE6fPo2cnBz5+QcffBBPPfUUDhw4gJEjR8JoNGLmzJlYvXo1du7ciRkzZuDSSy9FSUlJpz/nb3/7G6666ir8/PPPmDlzJubOnYuamhqP1zc2NuK5557Du+++i59++gklJSW499575eeffvppvPfee1i2bBnWr18Pg8GA5cuXd3oPR48exYwZM3DFFVfg559/xkcffYR169bhjjvucLnu2WefxahRo7Bz5048+OCDuOuuu7Bq1SoAgM1mw6xZs1BTU4M1a9Zg1apVKCoqwtVXXy2//uuvv8acOXMwc+ZM7Ny5E6tXr8bEiRNdfsbf//53jB8/Hjt37sTtt9+OP/zhDzh48GCn909EXejxuelEFLHOPfdc8a677nJ57IcffhABiMuXL+/y9cOGDRNffvll+fu+ffuKL7zwgvw9APGvf/2r/L3RaBQBiCtWrHD5WbW1taIoiuKyZctEAOKRI0fk1yxdulTU6/Xy93q9Xnz22Wfl7y0Wi5ibmyvOmjXL433efPPN4vz5810eW7t2rahQKMSmpib53mfMmOFyzdVXXy1edNFFoiiK4rfffisqlUqxpKREfn7fvn0iAHHLli2iKIpiQUGBOHfuXI/30bdvX/G6666Tv7fZbGJ6err46quvenwNEXWNlR0i6pbx48e7fG80GnHvvfdiyJAhSExMRFxcHA4cONBlZWfkyJHyn2NjYxEfH4+KigqP18fExKB///7y95mZmfL19fX1KC8vd6mWKJVKjBs3rtN72L17N9566y3ExcXJX9OnT4fNZkNxcbF8XUFBgcvrCgoKcODAAQDAgQMHkJOT41L9Gjp0KBITE+Vrdu3ahQsuuKDTe3H+PARBQEZGRqefBxF1jQ3KRNQt7XdV3XvvvVi1ahWee+45DBgwANHR0bjyyithNps7fZ+oqCiX7wVBgM1m8+l6URR9vHtXRqMRv//973HnnXd2eC43N7dH7+0sOjq6y2t8/TyIqGus7BCRR2q1Glar1atr169fjxtuuAFz5szBiBEjkJGRgWPHjvn3BttJSEiAXq/H1q1b5cesVit27NjR6evGjh2L/fv3Y8CAAR2+1Gq1fN2mTZtcXrdp0yYMGTIEADBkyBCUlpaitLRUfn7//v2oq6vD0KFDAdirNqtXr+7x35OIfMPKDhF5lJeXh82bN+PYsWOIi4tDcnKyx2sHDhyITz/9FJdeeikEQcDDDz8clIrEH//4RyxevBgDBgzA4MGD8fLLL6O2thaCIHh8zQMPPIDJkyfjjjvuwC233ILY2Fjs378fq1atwiuvvCJft379ejzzzDOYPXs2Vq1ahY8//hhff/01AKCwsBAjRozA3LlzsWTJElgsFtx+++0499xz5SW/Rx55BBdccAH69++Pa665BhaLBf/73//wwAMP+PdDITrDsbJDRB7de++9UCqVGDp0KNLS0jrtv3n++eeRlJSEKVOm4NJLL8X06dMxduzYAN6t3QMPPIBrr70W119/PQoKCuT+G61W6/E1I0eOxJo1a3Do0CGcffbZGDNmDBYtWoSsrCyX6/70pz9h27ZtGDNmDB5//HE8//zzmD59OgD7ctPnn3+OpKQknHPOOSgsLES/fv3w0Ucfya8/77zz8PHHH+OLL77A6NGjcf7552PLli3++SCISCaIPV3sJiIKYTabDUOGDMFVV12Fxx57rNvvk5eXh4ULF2LhwoW9d3NEFBBcxiKiiHL8+HF8++23OPfcc9HS0oJXXnkFxcXF+O1vfxvsWyOiIOEyFhFFFIVCgbfeegsTJkzA1KlTsWfPHnz33XdyIzERnXm4jEVEREQRjZUdIiIiimgMO0RERBTRGHaIiIgoojHsEBERUURj2CEiIqKIxrBDREREEY1hh4iIiCIaww4RERFFtP8PXY5Obj6sAjQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 为对比学习负采样准备词频率分布\n",
    "vocab_size = len(dataset.token2id)\n",
    "embed_size = 128\n",
    "distribution = dataset.get_word_distribution()\n",
    "print(distribution)\n",
    "model = SkipGramNCE(vocab_size, embed_size, distribution)\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "\n",
    "# 定义静态方法collate_batch批量处理数据，转化为PyTorch可以需要的张量类型\n",
    "class DataCollator:\n",
    "    @classmethod\n",
    "    def collate_batch(cls, batch):\n",
    "        batch = np.array(batch)\n",
    "        input_ids = torch.tensor(batch[:, 0], dtype=torch.long)\n",
    "        labels = torch.tensor(batch[:, 1], dtype=torch.long)\n",
    "        return {'input_ids': input_ids, 'labels': labels}\n",
    "\n",
    "# 定义训练参数以及训练循环\n",
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 1e-3\n",
    "epoch_loss = []\n",
    "\n",
    "data_collator = DataCollator()\n",
    "dataloader = DataLoader(data, batch_size=batch_size, shuffle=True,\\\n",
    "    collate_fn=data_collator.collate_batch)\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "model.zero_grad()\n",
    "model.train()\n",
    "\n",
    "# 需要提前安装tqdm\n",
    "from tqdm import trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 训练过程，每步读取数据，送入模型计算损失，并使用PyTorch进行优化\n",
    "with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "    for epoch in pbar:\n",
    "        for step, batch in enumerate(dataloader):\n",
    "            loss = model(**batch)\n",
    "            pbar.set_description(f'epoch-{epoch}, loss={loss.item():.4f}')\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "        epoch_loss.append(loss.item())\n",
    "    \n",
    "epoch_loss = np.array(epoch_loss)\n",
    "plt.plot(range(len(epoch_loss)), epoch_loss)\n",
    "plt.xlabel('training epoch')\n",
    "plt.ylabel('loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9430e9a",
   "metadata": {},
   "source": [
    "TF-IDF加权\n",
    "\n",
    "定义词频率（term frequency）。注意到不同长度的文章词频率会有较大差距，不利于比较和运算，因此可以对词频率取对数。\n",
    "\n",
    "$$\\text{tf}_{t,d} = \\log (\\text{count}(t,d) + 1)$$\n",
    "\n",
    "其中$\\text{count}(t,d)$表示词$t$在文档$d$中出现的次数，为了避免对0取对数，把所有的计数加1。\n",
    "\n",
    "那么如何区分高频词与低频词呢？TF-IDF引入了另一个重要的评价指标——文档频率（document frequency），即一个词在语料库所包含的多少篇文档中出现。在所有文档里出现的词往往是虚词或是常见实词，而只在少量文档里出现的词往往是具有明确含义的实词并且具有很强的文档区分度。用$\\text{df}_t$来表示在多少篇文档中出现了词$t$。\n",
    "\n",
    "为了压低高频词和提升低频词的影响，TF-IDF使用文档频率的倒数，也就是逆向文档频率（inverse document frequency）来对词频率进行加权。这很好理解，一个词的文档频率越高，其倒数就越小，权重就越小。\n",
    "\n",
    "$$\\text{idf}_t = \\log \\frac{N}{\\text{df}_t}$$\n",
    "\n",
    "其中$N$表示文档总数。为了避免分母为0，通常会将分母改为$\\text{df}_t+1$。\n",
    "\n",
    "基于词频率和逆向文档频率，得到TF-IDF的最终值为：\n",
    "\n",
    "$$w_{t,d} = \\text{tf}_{t,d} \\times \\text{idf}_{t}$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f765e353",
   "metadata": {},
   "source": [
    "很多情况下会额外对文档的TF-IDF向量使用L2归一化，使得不同文档的TF-IDF向量具有相同的模长，便于相互比较。\n",
    "下面给出了TF-IDF的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9ce8e610",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFIDF:\n",
    "    def __init__(self, vocab_size, norm='l2', smooth_idf=True,\\\n",
    "                 sublinear_tf=True):\n",
    "        self.vocab_size = vocab_size\n",
    "        self.norm = norm\n",
    "        self.smooth_idf = smooth_idf\n",
    "        self.sublinear_tf = sublinear_tf\n",
    "    \n",
    "    def fit(self, X):\n",
    "        doc_freq = np.zeros(self.vocab_size, dtype=np.float64)\n",
    "        for data in X:\n",
    "            for token_id in set(data):\n",
    "                doc_freq[token_id] += 1\n",
    "        doc_freq += int(self.smooth_idf)\n",
    "        n_samples = len(X) + int(self.smooth_idf)\n",
    "        self.idf = np.log(n_samples / doc_freq) + 1\n",
    "    \n",
    "    def transform(self, X):\n",
    "        assert hasattr(self, 'idf')\n",
    "        term_freq = np.zeros((len(X), self.vocab_size), dtype=np.float64)\n",
    "        for i, data in enumerate(X):\n",
    "            for token in data:\n",
    "                term_freq[i, token] += 1\n",
    "        if self.sublinear_tf:\n",
    "            term_freq = np.log(term_freq + 1)\n",
    "        Y = term_freq * self.idf\n",
    "        if self.norm:\n",
    "            row_norm = (Y**2).sum(axis=1)\n",
    "            row_norm[row_norm == 0] = 1\n",
    "            Y /= np.sqrt(row_norm)[:, None]\n",
    "        return Y\n",
    "    \n",
    "    def fit_transform(self, X):\n",
    "        self.fit(X)\n",
    "        return self.transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aaf5c264-4f5e-497a-89b4-5a1db61422f8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b25cb3fa-2c0c-420f-883d-e5902428c813",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5c41fbe-765b-4ba0-bfeb-a5c718dd8730",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
